# -*- coding: utf-8 -*-
from utils.operators.spark_submit import SparkSubmitOperator
from datetime import timedelta
from jms.dim import jms_dim__dim_lmdm_sys_first_code, jms_dim__dim_lmdm_sys_second_code, jms_dim__dim_lmdm_sys_network_expand, jms_dim__dim_lmdm_sys_network

from jms.dm.dm_cn_departure_transfer_timely_route_dt import jms_dm__dm_cn_departure_transfer_timely_route_dt
from jms.dm.dm_cn_departure_transfer_timely_route_dt_t2 import jms_dm__dm_cn_departure_transfer_timely_route_dt_t2
from jms.dm.dm_waybill_prescription_reach_details import jms_dm__dm_waybill_prescription_reach_details_dt
from jms.ods import jms_ods__tab_barscan_loading__0730, jms_ods__tab_barscan_unloading__0730, jms_ods__tab_barscan_centerarrival__0730, jms_ods__tab_barscan_centersend__0730, \
    jms_ods__tab_barscan_collect__0730, jms_ods__tab_barscan_sign__0730, jms_ods__tab_barscan_taking, jms_ods__tab_barscan_warehousing, jms_ods__tab_barscan_sitesend, jms_ods__tab_barscan_sitearrival, \
    jms_ods__tab_end_piece, jms_ods__yl_tms_branch_shipment__0730, jms_ods__yl_tmsnew_branch_shipment__0730, jms_ods__yl_tmsnew_branch_shipment_stop__0730, jms_ods__yl_tmsnew_tms_shipment__0730, \
    jms_ods__yl_oms_oms_waybill, jms_ods__yl_oms_interceptorpiece__0730, jms_ods__registration_problem_piece
from jms.ods.tms.yl_tmsnew_tms_track__0730 import jms_ods__yl_tmsnew_tms_track__0730

jms_dm__dm_cn_sham_deliver_goods_dt = SparkSubmitOperator(
    task_id='jms_dm__dm_cn_sham_deliver_goods_dt',
    pool_slots=16,
    pool='unlimited_pool',
    # task_concurrency=1,  # 如果任务不支持并发，则将 task_concurrency 设为 1
    name='jms_dm__dm_cn_sham_deliver_goods_dt_{{ execution_date | date_add(-1) | cst_ds }}',  # yarn 任务名称
    driver_memory='16G',
    executor_memory='16G',
    executor_cores=8,
    driver_cores=8,
    num_executors=90,
    execution_timeout=timedelta(hours=1),
    retries=0,
    email=['zhangqinglin@jtexpress.com','yl_bigdata@yl-scm.com'],
    conf={'spark.executor.memoryOverhead': 8192,
          'spark.core.connection.ack.wait.timeout': 3000,
          'spark.default.parallelism': 6000,
          'spark.locality.wait': 60,
          'spark.driver.maxResultSize': 20,
          'spark.sql.shuffle.partitions': 2000
          },
    java_class='com.yunlu.bigdata.jobs.export.ExportShamDeliverGoods',  # spark 主类
    application='hdfs:///scheduler/jms/spark/lyx/sham_deliver_goods/jobs-1.0-SNAPSHOT-jar-with-dependencies.jar',
    # spark jar 包
    application_args=['{{ execution_date | date_add(-1) | cst_ds }}', '{{ execution_date | date_add(-1) | cst_ds }}'],
    # 参数dt 2020-10-26
)

# 设置依赖
jms_dm__dm_cn_sham_deliver_goods_dt << [
    jms_dm__dm_cn_departure_transfer_timely_route_dt,  # 逻辑上无关联，调整并行度
    jms_dm__dm_cn_departure_transfer_timely_route_dt_t2,
    jms_ods__tab_barscan_loading__0730,
    jms_ods__tab_barscan_unloading__0730,
    jms_ods__tab_barscan_centerarrival__0730,
    jms_ods__tab_barscan_centersend__0730,
    jms_ods__tab_barscan_collect__0730,
    jms_ods__tab_barscan_sign__0730,
    jms_ods__tab_barscan_taking,
    jms_ods__tab_barscan_warehousing,
    jms_ods__tab_barscan_sitesend,
    jms_ods__tab_barscan_sitearrival,
    jms_ods__tab_end_piece,
    jms_ods__yl_tms_branch_shipment__0730,
    jms_ods__yl_tmsnew_tms_track__0730,
    jms_ods__yl_tmsnew_branch_shipment__0730,
    jms_ods__yl_tmsnew_branch_shipment_stop__0730,
    jms_ods__yl_tmsnew_tms_shipment__0730,
    jms_ods__yl_oms_interceptorpiece__0730,
    jms_ods__yl_oms_oms_waybill,
    jms_dim__dim_lmdm_sys_first_code,
    jms_dim__dim_lmdm_sys_second_code,
    jms_dim__dim_lmdm_sys_network_expand,
    jms_dim__dim_lmdm_sys_network,
    jms_dm__dm_waybill_prescription_reach_details_dt,
    jms_ods__registration_problem_piece,
]
